package com.mycampus.crawler.thu;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.model.HttpRequestBody;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.utils.HttpConstant;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ThuPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    private String urlPrefix = "http://career.cic.tsinghua.edu.cn";

    @Override
    public void process(Page page) {

        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        }

        String s = page.getUrl().get();
        System.out.println(s);


        if (s.endsWith("/xsglxt/f/jyxt/anony/xxfb")) {
            doProcessListPage(page);
        } else {
            doProcessDetail(page);
        }
    }

    private void doProcessDetail(Page page) {

        Html html = page.getHtml();
        String title = html.$("h1", "text").get();
        page.putField("title", title);

        String body = html.$("body").get();
        page.putField("content", body);
        page.putField("originUrl", page.getUrl().get());
    }

    private void doProcessListPage(Page page) {
        String pgno = page.getHtml().$("#pgno", "value").get();
        addNewListRequest(page, pgno);
        System.out.println("-----------" + pgno);
        addNewDetailRequests(page);
    }

    private void addNewDetailRequests(Page page) {
        List<String> all = page.getHtml().$("#todayList a", "ahref").all();
        System.out.println(all);
        for (String detailUrl : all) {
            addSingleDetailRequest(page, detailUrl);
        }
    }

    private void addSingleDetailRequest(Page page, String detailUrl) {
        String url = urlPrefix + detailUrl;
        page.addTargetRequest(url);
    }

    private void addNewListRequest(Page page, String pgno) {

        Integer pageNo = Integer.valueOf(pgno) + 1;
        Request request = new Request("http://career.cic.tsinghua.edu.cn/xsglxt/f/jyxt/anony/xxfb");
        request.setMethod(HttpConstant.Method.POST);
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("pgno", pageNo);
        request.setRequestBody(HttpRequestBody.form(paramMap, "utf-8"));

        page.addTargetRequest(request);
    }

    @Override
    public Site getSite() {
        return this.site;
    }

    public static void main(String[] args) {
        Request request = new Request("http://career.cic.tsinghua.edu.cn/xsglxt/f/jyxt/anony/xxfb");
        request.setMethod(HttpConstant.Method.POST);
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("pgno", "1");
        request.setRequestBody(HttpRequestBody.form(paramMap, "utf-8"));

        Spider.create(new ThuPageProcessor()).addPipeline(new JsonFilePipeline("/Users/qinyujie3")).addRequest(request).thread(1).run();
    }
}
